/*
 * Häufigkeiten der Wörter in Wikipedia-Datensatz
 */
package complet.app.tool.experiment;

import complet.helper.IOHelper;
import java.io.BufferedReader;
import java.io.BufferedWriter;
import java.io.IOException;
import java.util.Comparator;
import java.util.TreeMap;

/**
 *
 * @author paul
 */
public class experiment3_3 {

    private static BufferedReader wiki = IOHelper.openReadFile("E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\inputfiles\\wikipedia\\wiki.txt");
    private static BufferedWriter out1 = IOHelper.openWriteFile("E:\\paul\\Dokumente\\MyDocuments\\NetBeans Workspace\\Completion\\stats\\225_250T\\experiment3\\words_wiki.dat");
    private static TreeMap<String, Integer> tm = new TreeMap<String, Integer>();
    private static TreeMap<String, Integer> tm_s = null;
    private static final Comparator<String> COMP = new Comparator<String>() {

        public int compare(String s1, String s2) {
            if (tm.get(s1) > tm.get(s2)) {
                return -1;
            }
            return 1;
        }
    };

    public static void main(String args[]) {
        tm_s = new TreeMap<String, Integer>(COMP);
        try {
            String strLine;
            String line[];
            String word;
            int c = 0;
            while (true && (strLine = wiki.readLine()) != null) {
                c++;
                if (c % 10000 == 9999) {
                    IOHelper.pln("At " + c + " sentences");
                }
                if (c >= 5000000){
                    break;
                }

                line = strLine.toLowerCase().split(" ");
                for (int i = 0; i < line.length; i++) {
                    word = line[i];
                    int t = 0;
                    if (tm.containsKey(word)) {
                        t = tm.get(word);
                    }
                    tm.put(word, t + 1);
                }

            }

            IOHelper.pln("Sort things");
            tm_s.putAll(tm);

            IOHelper.pln("Ausgeben");
            for(String key: tm_s.keySet()){
                out1.write(key + "\t " + tm.get(key) + "\n");
            }
            out1.flush();
            out1.close();

        } catch (IOException ex) {
            IOHelper.logError(ex);
        }
    }
}
